/*
 * @Descripttion: 
 * @Author: 李增辉
 * @Date: 2024-03-01 17:29:03
 * @LastEditors: 李增辉
 * @LastEditTime: 2024-03-01 17:29:10
 */
const axios = require('axios')
const cheerio = require('cheerio')
const fs = require('fs')
const baseURL = 'https://www.gushiwen.cn/'
const startURL = 'default_1.aspx';
const crawler_Data = [];

const axios_instance = axios.create({
  baseURL,
  Headers: {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
  }
})

async function getPageData(url) {
  const { data } = await axios_instance.get(url);
  const $ = cheerio.load(data);
  $('.sons').each((index, item) => {
    //获取item下类名为cont的div
    const cont = $(item).find('.cont');
    if (cont.length === 0) return;
    const title = cont.find('b').text();
    const content = cont.find('.contson').text();
    crawler_Data.push({
      title,
      content
    })
  });

}

async function init() {
  promises = []
  for (let i = 1; i < 5; i++) {
    promises.push(getPageData(`default_${i}.aspx`))
  }
  await Promise.all(promises)
  fs.writeFileSync('./data.json', JSON.stringify(crawler_Data))
}

init()
